#include "stdafx.h"
#include "CpUtil.h"
#include "Tokenizer.h"
#include "assert.h"

using namespace CppClean;

const static char* s_Letters = "abcdefghijklmnopqrstuvwxyz";			//Letters
const static char* s_UpplerLetters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ";		//Upper Letters
const static char* s_DigitalInNames = "_0123456789$";					//Signs can occurred in Identifier
const static char* s_Digitals = "01234567890eE-+";						//Validate digital except '.', we will add it independently.
const static char* s_Hexs = "0123456789abcdefABCDEF";					//Hex digital.

CTokenizer::CTokenizer(const char* pData, DWORD dwSize)
:m_szSource(NULL), m_dwEndPos(0), m_dwStart(0), m_dwCurrent(0), m_bIgnoreErrors(FALSE), m_nCountIfs(0)
{
	CCharUtil::AppendSet(m_aIdentifies, s_Letters);
	CCharUtil::AppendSet(m_aIdentifies, s_UpplerLetters);
	CCharUtil::AppendSet(m_aIdentifies, s_DigitalInNames);

	CCharUtil::AppendSet(m_aHex, s_Hexs);
	CCharUtil::AppendSet(m_aDigits, s_Digitals);
	CCharUtil::AppendSet(m_aDigits2, s_Digitals);
	CCharUtil::AppendSet(m_aDigits2, ".");

	m_aStringPrefixs.insert("R");
	m_aStringPrefixs.insert("u8");
	m_aStringPrefixs.insert("u8R");
	m_aStringPrefixs.insert("u");
	m_aStringPrefixs.insert("uR");
	m_aStringPrefixs.insert("U");
	m_aStringPrefixs.insert("UR");
	m_aStringPrefixs.insert("L");
	m_aStringPrefixs.insert("LR");

	m_szSource = (char*)pData;
	m_dwEndPos = dwSize;
	/*
	CCharUtil::DumpSet(m_aDigits2);
	CCharUtil::DumpSet(m_aDigits);
	CCharUtil::DumpSet(m_aHex);
	CCharUtil::DumpSet(m_aIdentifies);
	if(m_szSource)
		OutputDebugStringA(m_szSource);
	*/
}

CTokenizer::~CTokenizer(void)
{
}

bool CTokenizer::IsStringPrefix(const char* s, int start, int end)
{
	string sTest(s + start, end - start);
	STRINGSET::iterator it = m_aStringPrefixs.find(sTest);
	return it != m_aStringPrefixs.end();
}

//Test sample: char s = ''; char s='\\'; char s='\'';
int CTokenizer::GetChar(int start, int curr)
{
	int i = curr;
	i = CCharUtil::findchar(m_szSource, '\'', i + 1);

	//# Count the trailing backslashes. 
	while(m_szSource[i-1] == '\\')
	{
		if(i - 2 > start && m_szSource[i - 2] == '\\')
			break;

		i = CCharUtil::findchar(m_szSource, '\'', i + 1);
		if(i < 0)
			i = start;
	}
	return i + 1;
}

//Test sample: const char* s = L"any string \\\" Most \\Host";
int CTokenizer::GetString(int start, int curr)
{
	int i = curr;
	i = CCharUtil::findchar(m_szSource, '"', i + 1);

	//# Count the trailing backslashes. 
	while(m_szSource[i-1] == '\\')
	{
		int blackslash_count = 1;
		int j = i - 2;
		while(j > start && m_szSource[j] == '\\')
		{
			++blackslash_count;
			j -= 1;
		}
		//When trailing backslashes are even, they escape each other.
		if(blackslash_count % 2 == 0)
			break;

		i = CCharUtil::findchar(m_szSource, '"', i + 1);
	}
	return i + 1;
}

CToken* CTokenizer::GetNextToken()
{
	DWORD& i = m_dwCurrent;
	char* source = m_szSource;

	while(i < m_dwEndPos)
	{
		while(i < m_dwEndPos && isspace(m_szSource[m_dwCurrent]))
			i += 1;
		if(i >= m_dwEndPos)
			return NULL;

		int start = i;
		char c = m_szSource[i];
		CToken::TokenType token_type = CToken::eTTUnknown;
		if(isalpha(c) || c == '_')
		{
			token_type = CToken::eTTName;
			while(CCharUtil::InSet(source[i], m_aIdentifies))
				i += 1;
			// String and character constants can look like a name if
			// they are something like L"".
			// u, U, and L are valid C++0x character preffixes.
			if(source[i] == '\'' && (i - start) == 1 &&	
				(c == 'u' || c == 'U' || c== 'L'))
			{
				token_type = CToken::eTTConstant;
				i = GetChar(start, i);
			}
			else if(source[i] == '"' && IsStringPrefix(source, start, i))
			{
				token_type = CToken::eTTConstant;
				i = GetString(start, i);
			}
		}
		else if(c == '/' && source[i+1] == '/')    //Find // comments.
		{
			i = CCharUtil::findchar(source, '\n', i);
			if(i == -1)  // Handle EOF.
				i = m_dwEndPos;
			continue;
		}
		else if(c == '/' && source[i+1] == '*')    // Find /* comments. */
		{
			i = CCharUtil::findchar(source, "*/", i) + 2;
			continue;
		}
		// : or :: (plus other chars).
		else if(CCharUtil::findchar(":+-<>&|*=", c, 0) != -1)
		{
			token_type = CToken::eTTSyntax;
			i += 1;
			char new_ch = source[i];
			if(new_ch == c)	//::, <<, >>, ++, --, ==, &&, ||
			{
				i += 1;
				if(source[i] == '=') //handle >>=;<<=
					i++;
			}
			else if(c == '-' && new_ch == '>')	//->, <>
				i += 1;
			else if(new_ch == '=')	// -=, +=, *=, |=, &=, >=, <=
				i += 1;
		}
		else if(CCharUtil::findchar("()[]{}~!?^%;/.,", c, 0) != -1) //Handle single char tokens.
		{
			token_type = CToken::eTTSyntax;
			i += 1;
			if(c == '.' && isdigit(source[i]))
			{
				token_type = CToken::eTTConstant;
				i += 1;
				while(CCharUtil::InSet(source[i], m_aDigits))
					i++;
				//Handle float suffixes.
				if('l' == tolower(source[i]) || 'f' == towlower(source[i]))
					i++;
			}
			else if(source[i] == '=' && (c == '/' || c == '%' || c== '!' || c == '^'))	//Handle /=, %=, !=, ^=
			{
				i++;
			}
		}
		else if(isdigit(c))	//Find integer.
		{
			token_type = CToken::eTTConstant;
			if(c == '0' && tolower(source[i+1]) == 'x')//# Handle hex digits.
			{
				i += 2;
				while(CCharUtil::InSet(source[i], m_aHex))
					i += 1;
			}
			else
			{
				while(CCharUtil::InSet(source[i], m_aDigits2))
					i++;
			}
			//# Handle integer (and float) suffixes.
			char new_ch = tolower(source[i]);
			if(new_ch == 'f' || new_ch == 'u')	//'f', 'l'
				i++;
			else if(new_ch == 'l')	//'l'
			{
				i++;
				if(source[i] == 'l')	//'ll'
					i++;
			}
			else if(new_ch == 'u')	
			{
				if(source[i + 1] == 'l')	// 'ul'
				{
					i +=2;
					if(source[i] == 'l')	// 'ull'
						i++;
				}
			}
		}
		else if(c == '"')	//Find string.
		{
			token_type = CToken::eTTConstant;
			i = GetString(start, i);
		}
		else if(c == '\'')	//# Find char.
		{
			token_type = CToken::eTTConstant;
			i = GetChar(start, i);
		}
		else if(c == '#')	//Find pre-processor command.
		{
			token_type = CToken::eTTPreprocessor;
			string sTemp(source + i, 3);
			int count_ifs = 0;
			bool got_if = (sTemp.compare("#if") == 0 && isspace(source[i+3]));
			if(got_if)
			{
				count_ifs += 1;
			}
			else
			{
				string sEnd(source + i, 6);
				if(sEnd.compare("#endif") == 0)
					count_ifs -= 1;
			}
			if(count_ifs == 0)
				m_bIgnoreErrors = FALSE;

			//# TODO(nnorwitz): handle preprocessor statements (\ continuations).
			while(1)
			{
				int _i[5] = {0};
				_i[0] = CCharUtil::findchar(source, '\n', i);
				_i[1] = CCharUtil::findchar(source, "//", i);
				_i[2] = CCharUtil::findchar(source, "/*", i);
				_i[3] = CCharUtil::findchar(source, '"', i);
				_i[4] = m_dwEndPos;

				//# NOTE(nnorwitz): doesn't handle comments in #define macros.
				//# Get the first important symbol (newline, comment, EOF/end).
				i = m_dwEndPos;
				for(UINT x = 0; x < 5; x++)
				{
					if(_i[x] != -1 && _i[x] < i)
						i = _i[x];
				}
				//# Handle #include "dir//foo.h" properly.
				if(source[i] == '"')
				{
					i = CCharUtil::findchar(source, '"', i+1) + 1;
					assert(i > 0);
					continue;
				}
				//# Keep going if end of the line and the line ends with \.
				if(!(i == _i[0] && (source[i - 1] == '\\' || (source[i - 1] == '\r' && i >= 2 && source[i - 2] == '\\'))))
				{
					if(got_if)
					{
						string sCond(source + start + 4, i - start - 4);
						sCond.erase(0, sCond.find_first_not_of(" "));
						if(sCond.find_first_of("0") == 0 || sCond.find_first_of("(0)") == 0)
							m_bIgnoreErrors = TRUE;
					}
					break;
				}
				i+=1;
			}
		}
		else if(c == '\\') //Handle \ in code.//# This is different from the pre-processor \ handling.
		{
			i += 1;
			continue;
		}
		else if(m_bIgnoreErrors)
		{
			//# The tokenizer seems to be in pretty good shape.  This
			//# raise is conditionally disabled so that bogus code
			//# in an #if 0 block can be handled.  Since we will ignore
			//# it anyways, this is probably fine.  So disable the
			//# exception and  return the bogus char.
			i += 1;
		}
		else
		{
			int nPrev = i - 10 > 0 ? i - 10 : 0;
			int nNext = i + 10 < m_dwEndPos ? i + 10 : m_dwEndPos;
			string sContext(source + nPrev, nNext - nPrev);
			CDbgUtil::myDebugOutputA("Got invalid token in %s @ %d token:%s: %s\n", TRUE, "?", i, c, sContext.c_str());
			assert(false);
		}
		if(i <= 0)
		{
			CDbgUtil::myDebugOutputA("Invalid index, exiting now.", TRUE);
			return NULL;
		}
		string sName(source + start, i - start);
		return new CToken(token_type, sName.c_str(), start, i);
	}
	return NULL;
}

void CToken::OutputDebug()
{
	CDbgUtil::myDebugOutputA(GetString().c_str(), TRUE);
}

string CToken::GetString()
{
	string sType;
	switch(m_eType)
	{
	case eTTUnknown:
		sType = "Unknown";
		break;
	case eTTSyntax:
		sType = "Syntax";
		break;
	case eTTConstant:
		sType = "Constant";
		break;
	case eTTName:
		sType = "Name";
		break;
	case eTTPreprocessor:
		sType = "Preprocessor";
		break;
	default:
		break;
	}
	sType += ", ";
	sType += m_sName;

	return sType;
}

//////////////////////////////////////////////////////////////////////////
//
//////////////////////////////////////////////////////////////////////////

const static char* s_Types = "bool char int long short double float void wchar_t unsigned signed";
const static char* s_Modifiers = "auto register const inline extern static virtual volatile mutable";
const static char* s_Access = "public protected private friend";
const static char* s_Casts = "static_cast const_cast dynamic_cast reinterpret_cast";
const static char* s_Others = "true false asm class namespace using explicit this operator sizeof";
const static char* s_OtherTypes = "new delete typedef struct union enum typeid typename template";
const static char* s_Controls = "case switch default if else return goto";
const static char* s_Exceptions = "try catch throw";
const static char* s_Loops = "while do for break continue";

static CKeyword s_keyword;
//////////////////////////////////////////////////////////////////////////
//CKeyword
//////////////////////////////////////////////////////////////////////////
CKeyword::CKeyword()
{
	CStringUtil::Split(m_aAll, s_Types);
	CStringUtil::Split(m_aAll, s_Modifiers);
	CStringUtil::Split(m_aAll, s_Access);
	CStringUtil::Split(m_aAll, s_Casts);
	CStringUtil::Split(m_aAll, s_Others);
	CStringUtil::Split(m_aAll, s_OtherTypes);
	CStringUtil::Split(m_aAll, s_Controls);
	CStringUtil::Split(m_aAll, s_Exceptions);
	CStringUtil::Split(m_aAll, s_Loops);

	CStringUtil::Split(m_aBuiltIn, s_Types);
	CStringUtil::Split(m_aBuiltIn, s_Modifiers);
}

bool CKeyword::IsKeyword(const char* str)
{
	return CStringUtil::InSet(str, s_keyword.m_aAll);
}

bool CKeyword::IsBuiltInType(const char* str)
{
	if(strcmpi(str, "virtual") == 0 || strcmpi(str, "inline") == 0)
		return false;

	return CStringUtil::InSet(str, s_keyword.m_aBuiltIn);
}
